/*
Copyright (c) 2011, Stephane Albert
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the organization nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL STEPHANE ALBERT BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "debridFetcherBase.h"

debridFetcherBase::debridFetcherBase() {
    logLevel = CALL_LOG;
    maxRetries = 5;
    mustExit = false;
    mustStopDL = false;
    mustProcess = false;
    fileList = NULL;
    fetchThread = NULL;
    pthread_mutex_init(&mtx_fileList, NULL);
    pthread_mutex_init(&mtx_logMessage, NULL);
}

debridFetcherBase::~debridFetcherBase() {
    //Better be shure everything is dead before we free cURL
    stop();
    if (fileList != NULL)
        delete fileList;
    pthread_mutex_destroy(&mtx_fileList);
    pthread_mutex_destroy(&mtx_logMessage);
}

void debridFetcherBase::setMaxRetries(int newMaxRetries) {
    if (newMaxRetries < MAX_RETRIES)
        maxRetries = newMaxRetries;
    else
        maxRetries = MAX_RETRIES;
}

int debridFetcherBase::getMaxRetries() const {
    return maxRetries;
}

//FIXME need a better error handling
void debridFetcherBase::setLogFile(const std::string& newLogFile) {
    if (logFile.is_open())
        logFile.close();
    logFile.open(newLogFile.c_str(), std::ios_base::out|std::ios_base::app);
    if (!logFile.good())
        logFile.close();
}

void debridFetcherBase::setFailedLogFile(const std::string& newFailedLogFile) {
    if (logFailedFile.is_open())
        logFailedFile.close();
    logFailedFile.open(newFailedLogFile.c_str(), std::ios_base::out|std::ios_base::app);
    if (!logFailedFile.good())
        logFailedFile.close();
}

void debridFetcherBase::setLogLevel(int newLogLevel) {
    pthread_mutex_lock(&mtx_logMessage);
    if (logLevel > FALL_LOG)
        logLevel = FALL_LOG;
    else if (logLevel < 0)
        logLevel = NO_LOG;
    else
        logLevel = newLogLevel;
    pthread_mutex_unlock(&mtx_logMessage);
}

void debridFetcherBase::logMessage(const std::string& msg, int flags, bool addTime) {
    pthread_mutex_lock(&mtx_logMessage);
    if (logLevel) {
        int logMSB;
        std::ostream *curStream = NULL;
        //Get logLevel MSB to skip unused log options
        for (logMSB=FMSG_LOG; !(logMSB & logLevel); logMSB >>= 1);
        for (int curFlag=logMSB; curFlag != LFFAIL_LOG; curFlag >>= 1) {
            if (logLevel & curFlag) {
                if (flags & MISC_ERR) {     //Errors Flag
                    switch (curFlag) {
                        case CFAIL_LOG:
                            curStream = &std::cerr;
                            break;
                        case FFAIL_LOG:
                            curStream = &logFile;
                            break;
                        default:
                            continue;
                    }
                }
                else {                      //Standard output
                    switch (curFlag) {
                        case CMSG_LOG:
                            curStream = &std::cout;
                            break;
                        case FMSG_LOG:
                            curStream = &logFile;
                            break;
                        default:
                            continue;
                    }
                }
                if (addTime) {
                    time_t tstamp;
                    struct tm *localTime;
                    char timeBuffer[20];
                    time(&tstamp);
                    localTime = localtime(&tstamp);
                    strftime(timeBuffer, 20, _("%m/%d %H:%M:%S: "), localTime);
                    *curStream << timeBuffer;
                }
                if (flags == DECODE) {
                    *curStream << _("Decode: ");
                }
                else if (flags == DEBRID) {
                    *curStream << _("Debrid: ");
                }
                else if (flags == DOWNLOAD) {
                    *curStream << _("Downloading: ");
                }
                else if (flags == DECODE_ERR) {
                    *curStream << _("Error decoding: ");
                }
                else if (flags == DEBRID_ERR) {
                    *curStream << _("Error debrid: ");
                }
                else if (flags == DOWNLOAD_ERR) {
                    *curStream << _("Error downloading: ");
                }
                else if (flags == MISC_ERR) {
                    *curStream << _("Error: ");
                }
                *curStream << msg << std::endl;
                curStream->flush();
            }
        }
    }
    pthread_mutex_unlock(&mtx_logMessage);
}

void debridFetcherBase::logMessage(const std::string& msg, const std::string& linkURL, int flags, bool addTime) {
    pthread_mutex_lock(&mtx_logMessage);
    if (logLevel)
        if (flags & MISC_ERR)
            if (logLevel & LFFAIL_LOG)
                logFailedFile << linkURL << std::endl;
    std::string newMsg = linkURL + ", " + msg;
    pthread_mutex_unlock(&mtx_logMessage);
    logMessage(newMsg, flags, addTime);
}

void debridFetcherBase::setMonitorDir(const std::string& newMonitorDir) {
    if (!newMonitorDir.empty()) {
        monitorDirectory = newMonitorDir;
        if (newMonitorDir[newMonitorDir.size()-1] != DIR_SEP)
            monitorDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getMonitorDir() const {
    return monitorDirectory;
}

void debridFetcherBase::setWorkingDir(const std::string& newWorkingDir) {
    if (!newWorkingDir.empty()) {
        workingDirectory = newWorkingDir;
        if (newWorkingDir[newWorkingDir.size()-1] != DIR_SEP)
            workingDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getWorkingDir() const {
    return workingDirectory;
}

void debridFetcherBase::setDoneDir(const std::string& newDoneDir) {
    if (!newDoneDir.empty()) {
        doneDirectory = newDoneDir;
        if (newDoneDir[newDoneDir.size()-1] != DIR_SEP)
            doneDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getDoneDir() const {
    return doneDirectory;
}

size_t debridFetcherBase::dataToFileFFS(void *rawBuffer, size_t size, size_t nmemb, void *ffs_ptr) {
    fetchFileStruct *ffs = (fetchFileStruct *)ffs_ptr;
    if (ffs->fetcher->needToStop())
        return 0;
    return dataToFile(rawBuffer, size, nmemb, ffs->file);
}

std::string debridFetcherBase::decodeWuploadFolder(const std::string& folderURL) {  //Same as filesonic
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<caption>Files Folder</caption>");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<a href=\"", linksPos);
    if (curLinkPos == std::string::npos || curLinkPos > linksEndPos)
        return decodedURL;
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList(retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeMegauploadFolder(const std::string& folderID) {
    std::string folderURL = "http://www.megaupload.com/xml/folderfiles.php/?folderid=";
    std::string decodedURL;
    folderURL += folderID;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linkPos = retPage.find("url=\"");
    if (linkPos == std::string::npos)
        return decodedURL;
    linkPos += 5;
    size_t linkEndPos = retPage.find('"', linkPos);
    decodedURL = retPage.substr(linkPos, linkEndPos-linkPos);
    linkPos = retPage.find("url=\"", linkPos);
    while (linkPos != std::string::npos) {
        linkPos += 5;
        linkEndPos = retPage.find('"', linkPos);
        insertStrInFList(retPage.substr(linkPos, linkEndPos-linkPos));
        linkPos = retPage.find("url=\"", linkPos+1);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFileserveFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<table class=\"file_list\">");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<td class=\"m35\">", linksPos);
    if (curLinkPos == std::string::npos)
        return decodedURL;
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = "http://www.fileserve.com";
    decodedURL += retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<td class=\"m35\">", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList("http://www.fileserve.com"+retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<td class=\"m35\">", curLinkPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFilesonicFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<caption>Files Folder</caption>");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<a href=\"", linksPos);
    if (curLinkPos == std::string::npos)
        return decodedURL;
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList(retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
    }
    return decodedURL;
}

//FIXME due to insertion links are in reverse, maybe doing it top down next
std::string debridFetcherBase::decode4sharedFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t findEndPos;
    size_t findPos = retPage.find("&sId=");
    if (findPos == std::string::npos)
        return decodedURL;
    findPos += 5;
    findEndPos = retPage.find("\" language=\"", findPos);
    if (findEndPos == std::string::npos)
        return decodedURL;
    std::string sId = retPage.substr(findPos, findEndPos-findPos);
    findPos = retPage.find("<td class=\"filelistname\" >");
    std::string foundString;
    bool foundFirst = false;
    while (findPos != std::string::npos) {
        findPos = retPage.find("href=\"", findPos);
        findPos += 6;
        findEndPos = retPage.find('"', findPos);
        foundString = retPage.substr(findPos, findEndPos-findPos);
        if (foundString.find("javascript:openNewWindow('") != std::string::npos) {   //New link
            foundString.erase(0, 26);
            foundString.erase(foundString.length()-2, 2);
            if (!foundFirst) {
                decodedURL = foundString;
                foundFirst = true;
            }
            else
                insertStrInFList(foundString);
        }
        else if (foundString.find("javascript:changeDir(") != std::string::npos) {
            srand(time(NULL));
            foundString.erase(0, 21);
            foundString.erase(foundString.length()-1);
            std::string newFolder = "http://www.4shared.com/account/changedir.jsp?sId=";
            newFolder.append(sId);
            newFolder.append("&ajax=true&changedir=");
            newFolder.append(foundString);
            newFolder.append("&random=0.");
            newFolder.append(intToString(rand()));
            std::string newFolderPage = getData(newFolder);
            if (!newFolderPage.empty()) {
                size_t findFolderPos = newFolderPage.find("AccountFacade.currentDirPass");
                if (findFolderPos != std::string::npos) {
                    findFolderPos += 32;
                    size_t findFolderEndPos = newFolderPage.find("';", findFolderPos);
                    foundString = newFolderPage.substr(findFolderPos, findFolderEndPos-findFolderPos);
                    newFolder = "http://www.4shared.com/dir/";
                    newFolder.append(foundString);
                    if (!foundFirst) {
                        decodedURL = decode4sharedFolder(newFolder);
                        foundFirst = true;
                    }
                    else
                        insertStrInFList(decode4sharedFolder(newFolder));
                }
            }
        }
        findPos = retPage.find("<td class=\"filelistname\" >", findEndPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFilecrop(const std::string& linkURL) {
    std::string decodedURL;
    std::string linkPage = getData(linkURL);
    //TODO find a way to bypass captcha
    size_t linkPos = linkPage.find("/captcha.php?id");
    if (linkPos != std::string::npos) {    //Captcha requested
        linkPage.insert(linkPos, "http://www.filecrop.com");
        std::string access = doneDirectory + "access.html";
        std::ofstream accessFile;
        accessFile.open(access.c_str());
        if (accessFile.is_open()) {
            accessFile << linkPage;
            std::string errMSg = _("unable to decode link, captcha is required check page in: ");
            errMSg += access;
            errMSg += _(".\nThen restart the download.");
            logMessage(errMSg, linkURL, MISC_ERR);
        }
        return linkURL;
    }
    linkPos = linkPage.find("<a class=\"linkmaincolor\"");
    if (linkPos != std::string::npos) {     //Single link
        linkPos += 58;
        size_t linkEndPos = linkPage.find('"', linkPos);
        if (linkEndPos != std::string::npos) {
            decodedURL = linkPage.substr(linkPos, linkEndPos-linkPos);
        }
    }
    else if ((linkPos = linkPage.find("<table class=clear>")) != std::string::npos) {   //Multiple links
        std::string curLink = "http://www.filecrop.com";
        bool firstPage = true;
        do {
            if (!firstPage) {
                --linkPos;
                size_t begLinkPos = linkPage.rfind('"', linkPos);
                ++begLinkPos;
                std::string newPageURL = "http://www.filecrop.com/"+linkPage.substr(begLinkPos, linkPos-begLinkPos+1);
                linkPage = getData(newPageURL);
                linkPos = linkPage.find("<table class=clear>");
            }
            do {
                linkPos = linkPage.find("\" href=\"", linkPos);
                linkPos += 8;
                curLink.resize(23);
                curLink += linkPage.substr(linkPos, linkPage.find('"', linkPos)-linkPos);
                if (firstPage) {
                    decodedURL = curLink;
                    firstPage = false;
                }
                else {
                    insertStrInFList(curLink);
                }
                linkPos = linkPage.find("<table class=clear>", linkPos);
            } while (linkPos != std::string::npos);
        } while ((linkPos = linkPage.find("\">Next</a></td>")) != std::string::npos);
    }
    else
        return linkURL;
    return decodedURL;
}

//TODO Turbobit folder retrieved are not ordered, we should do it.
std::string debridFetcherBase::decodeTurboBitFolder(const std::string& folderID) {
    std::string folderURL = "http://turbobit.net/downloadfolder/gridFile?id_folder=";
    std::string decodedURL;
    folderURL += folderID;
    size_t baseFolderURLLen = folderURL.length();
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t foundPos = retPage.find(",\"total\":");
    if (foundPos == std::string::npos)
        return decodedURL;
    foundPos += 9;
    size_t endFoundPos = retPage.find(",\"", foundPos);
    if (endFoundPos == std::string::npos)
        return decodedURL;
    unsigned short nbPages = stringToInt(retPage.substr(foundPos, endFoundPos-foundPos));
    std::string curLink = "http://turbobit.net/";
    for (unsigned short i=1; i <= nbPages; ++i) {
        if (i > 1) {
            folderURL.erase(baseFolderURLLen);
            folderURL.append("&page=");
            folderURL.append(intToString(i, 0));
            retPage = getData(folderURL);
            endFoundPos = 0;
        }
        while ((foundPos = retPage.find("{\"id\":\"", endFoundPos)) != std::string::npos) {
            foundPos += 7;
            endFoundPos = retPage.find("\",\"", foundPos);
            if (endFoundPos == std::string::npos)
                return decodedURL;
            //Remove previous link ID
            curLink.erase(20);
            curLink.append(retPage.substr(foundPos, endFoundPos-foundPos));
            curLink.append("html");
            if (decodedURL.empty())
                decodedURL = curLink;
            else
                insertStrInFList(curLink);
        }
    }
    return decodedURL;
}

//TODO FreakShare folder retrieved are not ordered, we should do it.
std::string debridFetcherBase::decodeFreakShareFolder(const std::string& folderURL) {
    std::string decodedURL;
    //Detect the folderID and MD5
    size_t foundPos = folderURL.find("folder/");
    if (foundPos == std::string::npos) {
        foundPos = folderURL.find("f_id=");
        if (foundPos == std::string::npos)
            return decodedURL;
        foundPos += 5;
    }
    else
        foundPos += 7;
    size_t endFoundPos = folderURL.find('/', foundPos);
    std::string folderID;
    if (endFoundPos == std::string::npos) {
        endFoundPos = folderURL.find('&', foundPos);
        if (endFoundPos == std::string::npos)
            return decodedURL;
        folderID = folderURL.substr(foundPos, endFoundPos-foundPos);
        foundPos = folderURL.find("f_md5=");
        if (foundPos == std::string::npos)
            return decodedURL;
        foundPos += 6;
        endFoundPos = folderURL.find('&', foundPos);
        if (endFoundPos == std::string::npos)
            return decodedURL;
    }
    else {
        folderID = folderURL.substr(foundPos, endFoundPos-foundPos);
        foundPos = endFoundPos+1;
        endFoundPos = folderURL.find('.', foundPos);
        if (endFoundPos == std::string::npos)
            return decodedURL;
    }
    std::string folderMD5 = folderURL.substr(foundPos, endFoundPos-foundPos);
    //std::string folderURL = "http://freakshare.com/?x=folder&f_id=158347&f_md5=6e46667&entrys=20&page=0&order=";
    std::string curURL = "http://freakshare.com/?x=folder&f_id=";
    curURL.append(folderID);
    curURL.append("&f_md5=");
    curURL.append(folderMD5);
    size_t baseFolderURLLen = curURL.length();
    std::string retPage = getData(curURL);
    if (retPage.empty())
        return decodedURL;
    foundPos = retPage.find(" - Pages:  ");
    if (foundPos == std::string::npos)
        return decodedURL;
    foundPos += 11;
    endFoundPos = retPage.find("\n", foundPos);
    if (endFoundPos == std::string::npos)
        return decodedURL;
    unsigned short nbPages = stringToInt(retPage.substr(foundPos, endFoundPos-foundPos));
    endFoundPos = 0;
    for (unsigned short i=1; i <= nbPages; ++i) {
        if (i > 1) {
            curURL.erase(baseFolderURLLen);
            curURL.append("&page=");
            curURL.append(intToString(i, 0));
            retPage = getData(curURL);
            endFoundPos = 0;
        }
        while ((foundPos = retPage.find("<span class=\"f_name\"><a href=\"", endFoundPos)) != std::string::npos) {
            foundPos += 30;
            endFoundPos = retPage.find("\" target=\"_blank\">", foundPos);
            if (endFoundPos == std::string::npos)
                return decodedURL;
            if (decodedURL.empty())
                decodedURL = retPage.substr(foundPos, endFoundPos-foundPos);
            else
                insertStrInFList(retPage.substr(foundPos, endFoundPos-foundPos));
        }
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeBitShareFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<table style=\"width:100%;\" class=\"basic\">");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<a href=\"", linksPos);
    if (curLinkPos == std::string::npos)
        return decodedURL;
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList(retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeLink(const std::string& linkURL) {
    pthread_mutex_lock(&mtx_fileList);
    if (fileList == NULL)
        return linkURL;
    pthread_mutex_unlock(&mtx_fileList);
    std::string decodedURL;
    if (linkURL.find("wupload.") != std::string::npos) {
        if (linkURL.find("/folder/") != std::string::npos)
            decodedURL = decodeWuploadFolder(linkURL);
        else
            decodedURL = linkURL;
        size_t remPos = decodedURL.find("/w867/");
        if (remPos != std::string::npos)
            decodedURL.erase(remPos, 5);
        return decodedURL;
    }
    else if (linkURL.find("turbobit.") != std::string::npos) {
        size_t folderPos = linkURL.find("/download/folder/");
        if (folderPos != std::string::npos)
            decodedURL = decodeTurboBitFolder(linkURL.substr(folderPos+17));
        else
            decodedURL = linkURL;
        return decodedURL;
    }
    else if (linkURL.find("freakshare.") != std::string::npos) {
        size_t folderPos = linkURL.find("/folder/");
        if (folderPos == std::string::npos) {
            folderPos = linkURL.find("x=folder");
            if (folderPos == std::string::npos)
                decodedURL = linkURL;
            else
                decodedURL = decodeFreakShareFolder(linkURL);
        }
        else
            decodedURL = decodeFreakShareFolder(linkURL);
        return decodedURL;
    }
    else if (linkURL.find("bitshare.") != std::string::npos) {
        size_t folderPos = linkURL.find("/?d=");
        if (folderPos != std::string::npos)
            decodedURL = decodeBitShareFolder(linkURL);
        else
            decodedURL = linkURL;
        return decodedURL;
    }
    else if (linkURL.find("ul.to") != std::string::npos) {
        size_t linkPos = linkURL.find("ul.to/");
        if (linkPos == std::string::npos)
            return linkURL;
        linkPos += 6;
        decodedURL = "http://uploaded.to/file/";
        decodedURL.append(linkURL.substr(linkPos));
        return decodedURL;
    }
    else if (linkURL.find("megaupload.") != std::string::npos) {
        size_t folderPos = linkURL.find("/?f=");
        if (folderPos != std::string::npos)
            decodedURL = decodeMegauploadFolder(linkURL.substr(folderPos+4));
        else
            decodedURL = linkURL;
        return decodedURL;
    }
    else if (linkURL.find(".rapidshare.") != std::string::npos) {
        size_t linkPos = linkURL.find("#!download");    //New version, AD hates it !
        if (linkPos == std::string::npos)
            return linkURL;
        decodedURL = "https://rapidshare.com/files/";
        linkPos = linkURL.find('|', linkPos);
        linkPos = linkURL.find('|', linkPos+1);
        ++linkPos;
        size_t linkEndPos = linkURL.find('|', linkPos+1);
        decodedURL.append(linkURL.substr(linkPos, linkEndPos-linkPos));
        decodedURL.append("/");
        linkPos = linkEndPos;
        ++linkPos;
        linkEndPos = linkURL.find('|', linkPos);
        decodedURL.append(linkURL.substr(linkPos, linkEndPos-linkPos));
        return decodedURL;
    }
    else if (linkURL.find("fileserve.") != std::string::npos) {
        if (linkURL.find("/list/") != std::string::npos)
            decodedURL = decodeFileserveFolder(linkURL);
        else
            decodedURL = linkURL;
        return decodedURL;
    }
    else if (linkURL.find("4shared.") != std::string::npos) {
        if (linkURL.find("/dir/") != std::string::npos)
            decodedURL = decode4sharedFolder(linkURL);
        else
            decodedURL = linkURL;
        return decodedURL;
    }
    else if (linkURL.find("filesonic.") != std::string::npos) {
        if (linkURL.find("/folder/") != std::string::npos)
            decodedURL = decodeFilesonicFolder(linkURL);
        else
            decodedURL = linkURL;
        size_t remPos = decodedURL.find("/file/r");
        if (remPos != std::string::npos)
            decodedURL.erase(remPos+5, 11);
        return decodedURL;
    }
    else if (linkURL.find("filecrop.") != std::string::npos) {
        decodedURL = decodeFilecrop(linkURL);
    }
    else if (linkURL.find("filestube.") != std::string::npos) {
        std::string linkPage = getData(linkURL);
        size_t linksPos = linkPage.find("id=\"copy_paste_links\"");
        if (linksPos != std::string::npos) {
            linksPos = linkPage.find("\">", linksPos);          //We focus on the end of the tag, normaly that's where the links are

            size_t linksEndPos = linkPage.find("</pre> </div>", linksPos);
            if (linksEndPos == std::string::npos)
                return decodedURL;
            else
                linksPos += 2;
            size_t curLinkEndPos = linkPage.find("http://", linksPos+1);
            if (curLinkEndPos > linksEndPos)                    //Another http link not found in the list search for https
                curLinkEndPos = linkPage.find("https://", linksPos+1);
            if (curLinkEndPos > linksEndPos) {  //Only one link
                decodedURL = linkPage.substr(linksPos, linksEndPos-linksPos);
            }
            else {                              //Mutliple links
                decodedURL = linkPage.substr(linksPos, curLinkEndPos-linksPos);
                linksPos = curLinkEndPos;
                insertStrInFList(linkPage.substr(linksPos, linksEndPos-linksPos));
            }
            for (int i=decodedURL.length()-1; decodedURL[i] == '\r' || decodedURL[i] == '\n' || decodedURL[i] == ' '; --i) {    //Sanitize URL
                decodedURL.erase(i, decodedURL.length()-i);
            }
        }
        else if (linkPage.find("File no longer available") != std::string::npos) {
            return _("File no longer available");
        }
        else
            return decodedURL;
    }
    else if (linkURL.find("undeadlink.") != std::string::npos) {
        size_t idPos = linkURL.find("?id=");
        if (idPos != std::string::npos) {
            idPos += 4;
            size_t idEndPos = linkURL.find('&', idPos);
            decodedURL = "http://www.megaupload.com/?d=";
            if (idEndPos != std::string::npos)
                decodedURL.append(linkURL.substr(idPos, idEndPos-idPos));
            else
                decodedURL.append(linkURL.substr(idPos));
        }
        return decodedURL;
    }
    else if (linkURL.find("debridor.") != std::string::npos) {
        decodedURL = linkURL;
        if (decodedURL[7] == 'w')
            decodedURL.erase(7, 4);
        std::string linkPage = getData(decodedURL);
        decodedURL.clear();     //Empty for a blank response if it fails
        size_t linkPos = linkPage.find("<input name=\"get\"");
        if (linkPos != std::string::npos) {
            linkPos += 39;
            size_t linkEndPos = linkPage.find('"', linkPos);
            if (linkEndPos != std::string::npos) {
                std::string postStr = "get=";
                postStr.append(linkPage.substr(linkPos, linkEndPos-linkPos));
                postData("http://debridor.com/dl.php", postStr.c_str());
                char *newURL = NULL;
                if (curl_easy_getinfo(curlHandle, CURLINFO_REDIRECT_URL, &newURL) != CURLE_OK || newURL == NULL)
                        return decodedURL;
                decodedURL = newURL;
            }
        }
        else
            return decodedURL;
    }
    else if (linkURL.find("upsafe.org/") != std::string::npos) {
        decodedURL = linkURL;
        decodedURL.replace(linkURL.find("upsafe.org/"), 10, "debridor.com");
    }
    else if (linkURL.find("fuxx.us") != std::string::npos) {
        getData(linkURL, false);
        char *newURL = NULL;
        if (curl_easy_getinfo(curlHandle, CURLINFO_REDIRECT_URL, &newURL) != CURLE_OK || newURL == NULL)
                return decodedURL;
        decodedURL = newURL;
    }
    else
        return linkURL;

    decodedURL = decodeLink(decodedURL);
    return decodedURL;
}

int debridFetcherBase::getFile(const std::string& fileURL) {
    if (curlHandle == NULL)
        return CONN_ERR;
    //Log current download
    logMessage(fileURL, DOWNLOAD);
    //Extract filename from URL and open it
    size_t begPos = fileURL.rfind('/');
    if (begPos == std::string::npos)
        return URL_ERR;
    ++begPos;
    std::string filePath = workingDirectory;
    filePath.append(htmlDecode(fileURL.substr(begPos)));
    std::ofstream outFile(filePath.c_str(), std::ios_base::binary|std::ios_base::out);
    if (!outFile.is_open())
        return SYS_ERR;
    //Prepare struct to allow download canceling
    fetchFileStruct ffs;
    ffs.fetcher = this;
    ffs.file = &outFile;
    //Init cURL
    loadCURLSettings();
    curl_easy_setopt(curlHandle, CURLOPT_URL, fileURL.c_str());
    curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, dataToFileFFS);
    curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &ffs);
    curl_easy_setopt(curlHandle, CURLOPT_SEEKFUNCTION, seekFile);
    curl_easy_setopt(curlHandle, CURLOPT_SEEKDATA, &outFile);
    if (logLevel & CMSG_LOG)
        curl_easy_setopt(curlHandle, CURLOPT_NOPROGRESS, 0);

    //Stalled transfert detection
    curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_LIMIT, MIN_SPEED);               //Stalled Transfert detection 1Kb/s
    #if DEBUG > 0
        curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, 10);
    #else
        curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, MIN_SPEED_TIMEOUT);    //< 1 Kbps for 60 seconds
    #endif

    int funcRetCode = CONN_ERR;     //If we quit loop it's because something is wrong with the connection
    CURLcode retCode;
    std::streamoff resumePos=0;
    for (int retry=maxRetries; retry > 0; --retry) {
        retCode = curl_easy_perform(curlHandle);
        if (mustStopDL) {
            funcRetCode = CNCL_ERR;
            break;
        }
        /*
         * Stalled transfert check
         */
        else if (retCode == CURLE_OPERATION_TIMEDOUT) {
            resumePos = outFile.tellp();
            if (resumePos > 0) {        //Some bytes written, trying to resume
                logMessage(_("Transfert stalled, trying to resume."), MISC_ERR);
                curl_easy_setopt(curlHandle, CURLOPT_RESUME_FROM, resumePos);
                continue;
            }
        }
        /*
         * An error occurred
         */
        else if (retCode != CURLE_OK) {
            //Size checking (from response header)
            double contentLength;
            if (curl_easy_getinfo(curlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength) == CURLE_OK) {
                if (contentLength > 0) {        //If contentLength == -1 skip
                    if (outFile.tellp()-resumePos != contentLength) {     //We need to download more
                        resumePos = outFile.tellp();
                        logMessage(_("Transfert interrupted, trying to resume."), MISC_ERR);
                        curl_easy_setopt(curlHandle, CURLOPT_RESUME_FROM, resumePos);
                        continue;
                    }
                }
            }
            //If we can't set resuming position then we retry from the beginning
            resumePos = 0;
        }
        /*
         * No error
         */
        else {
            //Size checking
            double contentLength;
            funcRetCode = NO_ERR;
            if (curl_easy_getinfo(curlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength) == CURLE_OK) {
                if (contentLength > 0) {        //If contentLength == -1 skip
                    if (outFile.tellp()-resumePos != contentLength) {
                        std::string errorMessage = _("Size mismatch, expected: ");
                        errorMessage.append(doubleToString(contentLength+resumePos));
                        errorMessage.append(_(" found: "));
                        errorMessage.append(doubleToString(outFile.tellp()));
                        logMessage(errorMessage, MISC_ERR);
                        funcRetCode = SIZE_ERR;
                    }
                    else {		//If size match
                        //Close it so we flush buffers
                        outFile.close();
                        if (checkFileForError(filePath))
                            funcRetCode = HOST_ERR;
                    }
                }
            }
            break;
        }
        logMessage(_("Error during transfert, retrying."), MISC_ERR);
    }
    return funcRetCode;
}

void debridFetcherBase::dlEachFile() {
    logMessage(_("Downloading list."));
    std::string curFile;
    std::string decodedURL;
    std::string dlLink;
    std::string filePath;
    int retCode;
    size_t begPos;
    bool debridDone = false;
    pthread_mutex_lock(&mtx_fileList);
    while (fileList->good() && !mustExit) {
        getline(*fileList, curFile);
        pthread_mutex_unlock(&mtx_fileList);
        for (int i=curFile.length()-1; curFile[i] == '\r' || curFile[i] == '\n' || curFile[i] == ' '; --i) {    //Sanitize URL
            curFile.erase(i);
        }
        if (curFile.length() < 3) { //Possible new lines, dirty
            pthread_mutex_lock(&mtx_fileList);
            continue;
        }
        if (processServiceSpecific(curFile)) {
            pthread_mutex_lock(&mtx_fileList);
            continue;
        }
        if (!(debridDone = isDebrid(curFile))) {
            decodedURL = decodeLink(curFile);
            if (decodedURL.empty()) {
                logMessage(_("unhandled exception."), curFile, DECODE_ERR);
                pthread_mutex_lock(&mtx_fileList);
                continue;
            }
            else if (decodedURL.compare(0, 4, "http") || decodedURL.find("://", 4) > 5) {
                logMessage(decodedURL, curFile, DECODE_ERR);
                pthread_mutex_lock(&mtx_fileList);
                continue;
            }
        }

        for (int retry=maxRetries; retry > 0; --retry) {
            if (!debridDone) {
                dlLink = debridLink(decodedURL);
                if (dlLink.compare(0, 4, "http") || dlLink.find("://", 4) > 5) {     //If the URL doesn't begin with http*:// then it's an error
                    logMessage(dlLink, decodedURL, DEBRID_ERR);
                    dlLink.clear();
                    break;
                }
            }
            else {   //File is already ready -> direct download
                dlLink = curFile;
            }
            retCode = getFile(dlLink);
            if (retCode == HOST_ERR && !debridDone) {
                std::string errorMsg = _("server side error, trying to debrid the link again.");
                logMessage(errorMsg, DOWNLOAD_ERR);
            }
            else
                break;
        }

        //Debrid failed, skip
        if (dlLink.empty()) {
            pthread_mutex_lock(&mtx_fileList);
            continue;
        }

        filePath = workingDirectory;
        begPos = dlLink.rfind('/');
        ++begPos;
        filePath.append(htmlDecode(dlLink.substr(begPos)));

        if (retCode != NO_ERR) {
            std::string errorMsg;
            switch (retCode) {
                case URL_ERR:
                    errorMsg = _("incorrect URL.");
                    break;
                case CONN_ERR:
                    errorMsg = _("connection problem.");
                    break;
                case CNCL_ERR:
                    if (mustExit)
                        errorMsg = _("transfert cancelled, stopping thread.");
                    else
                        errorMsg = _("transfert cancelled.");
                    break;
                case SYS_ERR:
                    errorMsg = _("filesystem problem, unable to create temporary file.");
                    break;
                case HOST_ERR: {
                    if (debridDone)
                        errorMsg = _("server side error, maybe the link is too old.");
                    else
                        errorMsg = _("server side error, exceeded maximum attempt.");
                }
                    break;
                case SIZE_ERR:
                    errorMsg = _("size error.");
                    break;
            }

            //TODO depending on the error code keep the file as incomplete
            remove(filePath.c_str());

            if (debridDone)
                logMessage(errorMsg, curFile, DOWNLOAD_ERR);
            else
                logMessage(errorMsg, decodedURL, DOWNLOAD_ERR);
        }
        else {
            //Get new path
            std::string newFilePath;
            dupeResCode result = handleDupe(filePath, newFilePath);
            if (result != NO_DUPE) {
                std::string errorMsg;
                switch (result) {
                    case IN_ERR:
                        errorMsg = _("Error with the input \"");
                        errorMsg.append(filePath);
                        errorMsg.append(_("\" file: not found"));
                    break;
                    case DUP_FOUND:
                        errorMsg = _("Removed file, found duplicate at: ");
                        errorMsg.append(newFilePath);
                        remove(filePath.c_str());
                    break;
                    case NEW_NAME:
                        errorMsg = _("Renamed file to avoid overwritting to: ");
                        errorMsg.append(newFilePath);
                    case TOO_MANY:
                        errorMsg = _("Too many tries and still no filename available, keeping file in working directory.");
                    break;
                }
                logMessage(errorMsg, MISC_ERR);
            }
            if (!(result & ERROR)) {
                if (!moveFile(filePath, newFilePath)) {
                    /*std::cout << "debug mtx_lock: dlEachFile";
                    pthread_mutex_lock(&mtx_fileList);
                    std::cout << " ok" << std::endl;
                    continue;*/
                    logMessage(_("Unable to move file to donedir. Check workingdir."), MISC_ERR);
                }
            }
        }
        debridDone = false;
        //Be sure that we canceled what we wanted. And don't suppress the flag because the copy was too long for example.
        if (retCode == CNCL_ERR)
            mustStopDL = false;
        pthread_mutex_lock(&mtx_fileList);
    }
    delete fileList;
    fileList = NULL;
    pthread_mutex_unlock(&mtx_fileList);
    logMessage(_("Finished downloading list."));
}

void debridFetcherBase::addLink(const std::string& downloadLink) {
    pthread_mutex_lock(&mtx_fileList);
    if (fileList == NULL)
        fileList = new std::stringstream;
    pthread_mutex_unlock(&mtx_fileList);
    //FIXME Race condition
    insertStrInFList(downloadLink);
    if (fetchThread != NULL) {
        mustProcess = true;
    }
    else
        dlEachFile();
}

void debridFetcherBase::processDownloadList(const std::string& downloadList) {
    size_t listNamePos = downloadList.rfind(DIR_SEP);
    std::string newFilePath = workingDirectory;
    if (listNamePos != std::string::npos) {
        ++listNamePos;
        newFilePath.append(downloadList.substr(listNamePos));
    }
    else    //FIXME Possible Deadlock
        return;

    //Move the list in working dir
    moveFile(downloadList.c_str(), newFilePath);

    if (!processServiceSpecific(newFilePath)) {
        pthread_mutex_lock(&mtx_fileList);
        //If list already exists, append
        if (fileList != NULL) {
            std::stringstream *tempList = getFileContent(newFilePath);
            if (tempList)
                insertStrInFList(tempList->str());
        }
        //Else create it
        else {
            fileList = getFileContent(newFilePath);
        }
        //If filelist is good then process it
        if (fileList) {
            pthread_mutex_unlock(&mtx_fileList);
            dlEachFile();
        }
        pthread_mutex_lock(&mtx_fileList);
        delete fileList;
        fileList = NULL;
        pthread_mutex_unlock(&mtx_fileList);
        if (remove(newFilePath.c_str()) != 0) {
            logMessage(_("Unable to remove file, stopping thread."), MISC_ERR);
            mustExit = true;
        }
    }
}

std::stringstream *debridFetcherBase::getFileContent(const std::string& filePath) {
    if (filePath.empty())
        return NULL;
    std::ifstream inputFile(filePath.c_str());
    if (!inputFile.good())
        return NULL;
    std::stringstream *fileContent = new std::stringstream;
    *fileContent << inputFile.rdbuf();
    inputFile.close();
    return fileContent;
}

bool debridFetcherBase::checkDelBadFile(const std::string& fileName) {
    //Skips . and ..
    if (fileName.at(0) == '.') {    //Skipping hidden files and deleting the ones created by MacOS
        if (fileName == ".DS_Store") {            //Stupid MacOS .DS_Store
            std::string filePath = monitorDirectory + fileName;
            if (remove(filePath.c_str()) != 0) {
                //TODO maybe not fatal and just skip.
                logMessage(_("Unable to remove .DS_Store file, stopping thread."), MISC_ERR);
                mustExit = true;
            }
        }
        else if (fileName.length() > 1 && fileName.at(1) == '_') {         //MacOS generate ._ description files which tamper with the program behaviour
            std::string filePath = monitorDirectory + fileName;
            if (remove(filePath.c_str()) != 0) {
                logMessage(_("Unable to remove ._* file, stopping thread."), MISC_ERR);
                mustExit = true;
            }
        }
        return true;
    }
    return false;
}

dupeResCode debridFetcherBase::handleDupe(const std::string& fileName, std::string& outFileName, bool useMd5) const {
    outFileName.clear();
    if (fileName.empty())
        return IN_ERR;
    //Detect new filename
    size_t foundPos = fileName.rfind('/');
    if (foundPos == std::string::npos)
        return IN_ERR;
    ++foundPos;
    //Get inputFile size
    std::ifstream inputFile;
    std::streampos inFileLen;
    inputFile.open(fileName.c_str(), std::ios_base::binary);
    if (!inputFile.is_open())
        return IN_ERR;
    inputFile.seekg(0, std::ios_base::end);
    inFileLen = inputFile.tellg();
    inputFile.seekg(0, std::ios_base::beg);

    //Define where we want to create the new file
    outFileName = doneDirectory;
    outFileName.append(fileName.substr(foundPos));

    //Var used in the loop (so we gain speed)
    std::ifstream outputFile;
    std::streampos outFileLen;
    MD5 md5Hasher;
    char buffer[4096];
    std::streamsize readBytes;
    std::string md5Input;
    std::string md5Output;
    unsigned short streamIndex = 0;
    std::ifstream *curStream = &inputFile;

    std::string extension;
    //We search for the file extension
    foundPos = outFileName.rfind('.');
    if (foundPos != std::string::npos)
        extension = outFileName.substr(foundPos);
    else
        foundPos = outFileName.length();

    for (unsigned short fileNbr = 1; fileNbr < 257; ++fileNbr) {
        //Check if the outFileName exists
        outputFile.open(outFileName.c_str(), std::ios_base::binary);
        //If the file doesn't exist then we use this name
        if (!outputFile.is_open()) {
            if (fileNbr == 1)
                return NO_DUPE;
            else
                return NEW_NAME;
        }
        //Do the file has the same size ?
        outputFile.seekg(0, std::ios_base::end);
        outFileLen = outputFile.tellg();
        outputFile.seekg(0, std::ios_base::beg);
        //Same size, we should check if they are the same
        if (inFileLen == outFileLen) {
            //TODO we're using md5 to check the files but don't use the hashes, we should use them
            //or revert to using memcmp
            for (; streamIndex < 2; ++streamIndex) {
                while (curStream->good()) {
                    curStream->read(buffer, 4096);
                    readBytes = curStream->gcount();
                    md5Hasher.update(buffer, readBytes);
                }
                md5Hasher.finalize();
                if (streamIndex != 0)
                    md5Output = md5Hasher.hexdigest();
                else
                    md5Input = md5Hasher.hexdigest();
                md5Hasher.reset();
                curStream = &outputFile;
            }
            //Same files
            if (!md5Input.compare(md5Output))
                return DUP_FOUND;
        }
        //We've calculated the inputFile Hash, we can skip it now
        streamIndex = 1;
        inputFile.close();
        outputFile.close();
        outFileName.erase(foundPos);
        outFileName.append(".renamed_");
        outFileName.append(intToString(fileNbr, 3));
        //If no extension found we'll append the number at the end
        outFileName.append(extension);
    }
    return TOO_MANY;
}

void *debridFetcherBase::threadEntry(void *data) {
    ((debridFetcherBase*)data)->watchDirectory();
    return 0;
}

bool debridFetcherBase::createThread() {
    if (fetchThread == NULL) {
        fetchThread = new pthread_t;
        if (!pthread_create(fetchThread, NULL, threadEntry, (void*)this))
            return true;
    }
    return false;
}

bool debridFetcherBase::deleteThread() {
    if (fetchThread != NULL) {
        if (!pthread_join(*fetchThread, NULL)) {
            delete fetchThread;
            fetchThread = NULL;
            return true;
        }
    }
    return false;
}

bool debridFetcherBase::needToStop() {
    return mustStopDL;
}

void debridFetcherBase::insertStrInFList(const std::string& str) {
    std::stringstream endList;
    std::streampos curPos;
    pthread_mutex_lock(&mtx_fileList);
    curPos = fileList->tellg();
    if (fileList->eof())    //Removes EOF Flag
        fileList->clear();
    endList << fileList->rdbuf();   //If we are at the end it'll pass the eof flag to endList
    fileList->seekp(curPos, std::ios_base::beg);
    *fileList << str << std::endl;  //Finish with a newline to prepare for the next insertion
    if (endList.good())
        *fileList << endList.rdbuf();
    fileList->seekg(curPos, std::ios_base::beg);
    pthread_mutex_unlock(&mtx_fileList);
}

bool debridFetcherBase::start() {
    mustExit = false;
    mustStopDL = false;
    if (createThread()) {
        startServiceSpecific();
        return true;
    }
    return false;
}

bool debridFetcherBase::cancel() {
    mustStopDL = true;
    return true;
}

void debridFetcherBase::stop(bool forceExit) {
    if (fetchThread != NULL) {
        mustExit = true;
        if (forceExit)
            cancel();
        stopServiceSpecific();
        deleteThread();
    }
}
